from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader, PyPDFLoader
import wikipedia as wp
from langsmith import traceable


def wikipedia_loader(query: str):
    results = []
    for term in wp.search(query, results = 10):
        try:
            page = wp.page(term, auto_suggest=False)
            results.append({
                "page_content": page.summary,
                "type": "Document",
                "metadata": {"url": page.url}
            })
        except wp.DisambiguationError:
            pass

    print(results)


@traceable
def wikipedia_loader_langchain(query: str):
    docs = WikipediaLoader(query=query, load_max_docs=2).load()
    for doc in docs:
        print(doc)

def web_base_loader():
    loader = WebBaseLoader("https://www.espn.com/")
    docs = loader.load()
    for doc in docs:
        print(doc)


def pdf_loader():
    loader = PyPDFLoader("F:/tmp/遥远的救世主.pdf")
    documents = loader.load_and_split()

    for doc in documents:
        print(doc.metadata["page"])

if __name__ == "__main__":
    pdf_loader()